import requests
import traceback
import sys
import os
import time

from xjlibrary.our_file_dir import BaseDir

StartTime = None

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
sPath = BaseDir.get_new_path(TopPath, "download", "zswj",'download', "raw")
sUrl = r'http://www.libvideo.com/Search.aspx?page='
gClassName = 'OTHER'
ListSqls = list()
nCount = 0
rePage = 1
HEADERS = {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive',
           'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)', }


# 测试文字输出到文本
def output(value, files='log.txt'):
    f = open(files, 'w', encoding='utf-8')
    f.write(value)
    f.close()


# 保存目标站点
def Save(sPath):
    global nCount, rePage
    sMsg = None
    r = None
    rawUrl = r'http://www.libvideo.com/Search.aspx?page='
    # 下面的1177来自第一页最下面 看情况调整
    for i in range(rePage, 1178):
        if BaseDir.is_file_exists(os.path.join(sPath, str(i) + '.htm')):
            print("文件存在")
            continue
        url = rawUrl + str(i)
        try:
            r = requests.get(url, headers=HEADERS, timeout=15)
        except:
            sMsg = '*sMsg:' + traceback.format_exc()
        finally:
            if 'r' in locals() and r:
                r.close()

        if sMsg:
            print(sMsg)
            rePage = i
            return -1

        if r.status_code != requests.codes.ok:
            print('error-url:' + r.url)
            rePage = i
            return -1

        html = r.content

        if html.decode('utf-8').find('block mode') == -1:
            print('not find block mode')
            sys.exit(-1)

        if not html.decode('utf-8').strip().endswith('</html>'):
            output(html.decode('utf-8').strip())
            print('not find </html>')
            sys.exit(-1)

        with open(os.path.join(sPath, str(i) + '.htm'), 'wb') as f:
            f.write(html)
            nCount += 1
            print('写入文件数量：%d' % nCount)
            print('耗费时间: %d' % int(time.time() - StartTime) + '秒')
    rePage = 1
    return 0


# ----------------------------------------------逻辑部分-----------------------------------------------
# ----------------------------------------------录入数据库wanID----------------------------------------

def main():
    global StartTime
    StartTime = time.time()
    print('path:' + sPath)
    if not os.path.exists(sPath):
        os.makedirs(sPath)
    while 1:
        if 0 == Save(sPath):
            break


if __name__ == '__main__':
    main()

